1#!/usr/bin/env python
2
3r"""
4This module contains functions having to do with machine state: get_state,
5check_state, wait_state, etc.
6
7The 'State' is a composite of many pieces of data.  Therefore, the functions
8in this module define state as an ordered dictionary.  Here is an example of
9some test output showing machine state:
10
11default_state:
12  default_state[chassis]:                         On
13  default_state[boot_progress]:                   OSStart
14  default_state[operating_system]:                BootComplete
15  default_state[host]:                            Running
16  default_state[os_ping]:                         1
17  default_state[os_login]:                        1
18  default_state[os_run_cmd]:                      1
19
20Different users may very well have different needs when inquiring about
21state.  Support for new pieces of state information may be added to this
22module as needed.
23
24By using the wait_state function, a caller can start a boot and then wait for
25a precisely defined state to indicate that the boot has succeeded.  If
26the boot fails, they can see exactly why by looking at the current state as
27compared with the expected state.
28"""
29
30import gen_print as gp
31import gen_valid as gv
32import gen_robot_utils as gru
33import gen_cmd as gc
34import bmc_ssh_utils as bsu
35
36from robot.libraries.BuiltIn import BuiltIn
37from robot.utils import DotDict
38
39import re
40import os
41import sys
42import imp
43
44
45# NOTE: Avoid importing utils.robot because utils.robot imports state.py
46# (indirectly) which will cause failures.
47gru.my_import_resource("rest_client.robot")
48
49base_path = os.path.dirname(os.path.dirname(
50                            imp.find_module("gen_robot_print")[1])) + os.sep
51sys.path.append(base_path + "data/")
52
53# Previously, I had this coded:
54# import variables as var
55# However, we ran into a problem where a robot program did this...
56# Variables           ../../lib/ras/variables.py
57# Prior to doing this...
58# Library            ../lib/state.py
59
60# This caused the wrong variables.py file to be selected.  Attempts to fix this
61# have failed so far.  For the moment, we will hard-code the value we need from
62# the file.
63
64SYSTEM_STATE_URI = "/xyz/openbmc_project/state/"
65
66# The BMC code has recently been changed as far as what states are defined and
67# what the state values can be.  This module now has a means of processing both
68# the old style state (i.e. OBMC_STATES_VERSION = 0) and the new style (i.e.
69# OBMC_STATES_VERSION = 1).
70# The caller can set environment variable OBMC_STATES_VERSION to dictate
71# whether we're processing old or new style states.  If OBMC_STATES_VERSION is
72# not set it will default to 1.
73
74# As of the present moment, OBMC_STATES_VERSION of 0 is for cold that is so old
75# that it is no longer worthwhile to maintain.  The OBMC_STATES_VERSION 0 code
76# is being removed but the OBMC_STATES_VERSION value will stay for now in the
77# event that it is needed in the future.
78
79OBMC_STATES_VERSION = int(os.environ.get('OBMC_STATES_VERSION', 1))
80
81# When a user calls get_state w/o specifying req_states, default_req_states
82# is used as its value.
83default_req_states = ['rest',
84                      'chassis',
85                      'bmc',
86                      'boot_progress',
87                      'operating_system',
88                      'host',
89                      'os_ping',
90                      'os_login',
91                      'os_run_cmd']
92
93# valid_req_states is a list of sub states supported by the get_state function.
94# valid_req_states, default_req_states and master_os_up_match are used by the
95# get_state function.
96valid_req_states = ['ping',
97                    'packet_loss',
98                    'uptime',
99                    'epoch_seconds',
100                    'rest',
101                    'chassis',
102                    'requested_chassis',
103                    'bmc',
104                    'requested_bmc',
105                    'boot_progress',
106                    'operating_system',
107                    'host',
108                    'requested_host',
109                    'attempts_left',
110                    'os_ping',
111                    'os_login',
112                    'os_run_cmd']
113
114# valid_os_req_states and default_os_req_states are used by the os_get_state
115# function.
116# valid_os_req_states is a list of state information supported by the
117# get_os_state function.
118valid_os_req_states = ['os_ping',
119                       'os_login',
120                       'os_run_cmd']
121# When a user calls get_os_state w/o specifying req_states,
122# default_os_req_states is used as its value.
123default_os_req_states = ['os_ping',
124                         'os_login',
125                         'os_run_cmd']
126
127# Presently, some BMCs appear to not keep time very well.  This environment
128# variable directs the get_state function to use either the BMC's epoch time
129# or the local epoch time.
130USE_BMC_EPOCH_TIME = int(os.environ.get('USE_BMC_EPOCH_TIME', 0))
131
132# Useful state constant definition(s).
133# default_state is an initial value which may be of use to callers.
134default_state = DotDict([('rest', '1'),
135                         ('chassis', 'On'),
136                         ('bmc', 'Ready'),
137                         ('boot_progress', 'OSStart'),
138                         ('operating_system', 'BootComplete'),
139                         ('host', 'Running'),
140                         ('os_ping', '1'),
141                         ('os_login', '1'),
142                         ('os_run_cmd', '1')])
143
144# A match state for checking that the system is at "standby".
145standby_match_state = DotDict([('rest', '^1$'),
146                               ('chassis', '^Off$'),
147                               ('bmc', '^Ready$'),
148                               ('boot_progress', '^Off|Unspecified$'),
149                               ('operating_system', '^Inactive$'),
150                               ('host', '^Off$')])
151
152# A match state for checking that the system is at "os running".
153os_running_match_state = DotDict([('chassis', '^On$'),
154                                  ('bmc', '^Ready$'),
155                                  ('boot_progress',
156                                   'FW Progress, Starting OS|OSStart'),
157                                  ('operating_system', 'BootComplete'),
158                                  ('host', '^Running$'),
159                                  ('os_ping', '^1$'),
160                                  ('os_login', '^1$'),
161                                  ('os_run_cmd', '^1$')])
162
163# A master dictionary to determine whether the os may be up.
164master_os_up_match = DotDict([('chassis', '^On$'),
165                              ('bmc', '^Ready$'),
166                              ('boot_progress',
167                               'FW Progress, Starting OS|OSStart'),
168                              ('operating_system', 'BootComplete'),
169                              ('host', '^Running|Quiesced$')])
170
171invalid_state_match = DotDict([('rest', '^$'),
172                               ('chassis', '^$'),
173                               ('bmc', '^$'),
174                               ('boot_progress', '^$'),
175                               ('operating_system', '^$'),
176                               ('host', '^$')])
177
178
179def return_state_constant(state_name='default_state'):
180    r"""
181    Return the named state dictionary constant.
182    """
183
184    return eval(state_name)
185
186
187def anchor_state(state):
188    r"""
189    Add regular expression anchors ("^" and "$") to the beginning and end of
190    each item in the state dictionary passed in.  Return the resulting
191    dictionary.
192
193    Description of Arguments:
194    state    A dictionary such as the one returned by the get_state()
195             function.
196    """
197
198    anchored_state = state.copy()
199    for key, match_state_value in anchored_state.items():
200        anchored_state[key] = "^" + str(anchored_state[key]) + "$"
201
202    return anchored_state
203
204
205def strip_anchor_state(state):
206    r"""
207    Strip regular expression anchors ("^" and "$") from the beginning and end
208    of each item in the state dictionary passed in.  Return the resulting
209    dictionary.
210
211    Description of Arguments:
212    state    A dictionary such as the one returned by the get_state()
213             function.
214    """
215
216    stripped_state = state.copy()
217    for key, match_state_value in stripped_state.items():
218        stripped_state[key] = stripped_state[key].strip("^$")
219
220    return stripped_state
221
222
223def compare_states(state,
224                   match_state,
225                   match_type='and'):
226    r"""
227    Compare 2 state dictionaries.  Return True if they match and False if they
228    don't.  Note that the match_state dictionary does not need to have an entry
229    corresponding to each entry in the state dictionary.  But for each entry
230    that it does have, the corresponding state entry will be checked for a
231    match.
232
233    Description of arguments:
234    state           A state dictionary such as the one returned by the
235                    get_state function.
236    match_state     A dictionary whose key/value pairs are "state field"/
237                    "state value".  The state value is interpreted as a
238                    regular expression.  Every value in this dictionary is
239                    considered.  When match_type is 'and', if each and every
240                    comparison matches, the two dictionaries are considered to
241                    be matching.  If match_type is 'or', if any two of the
242                    elements compared match, the two dictionaries are
243                    considered to be matching.
244                    This value may also be any string accepted by
245                    return_state_constant (e.g. "standby_match_state").
246                    In such a case this function will call
247                    return_state_constant to convert it to a proper
248                    dictionary as described above.
249    match_type      This may be 'and' or 'or'.
250    """
251
252    error_message = gv.svalid_value(match_type, var_name="match_type",
253                                    valid_values=['and', 'or'])
254    if error_message != "":
255        BuiltIn().fail(gp.sprint_error(error_message))
256
257    try:
258        match_state = return_state_constant(match_state)
259    except TypeError:
260        pass
261
262    default_match = (match_type == 'and')
263    for key, match_state_value in match_state.items():
264        # Blank match_state_value means "don't care".
265        if match_state_value == "":
266            continue
267        try:
268            match = (re.match(match_state_value, str(state[key])) is not None)
269        except KeyError:
270            match = False
271
272        if match != default_match:
273            return match
274
275    return default_match
276
277
278def get_os_state(os_host="",
279                 os_username="",
280                 os_password="",
281                 req_states=default_os_req_states,
282                 os_up=True,
283                 quiet=None):
284    r"""
285    Get component states for the operating system such as ping, login,
286    etc, put them into a dictionary and return them to the caller.
287
288    Note that all substate values are strings.
289
290    Description of arguments:
291    os_host      The DNS name or IP address of the operating system.
292                 This defaults to global ${OS_HOST}.
293    os_username  The username to be used to login to the OS.
294                 This defaults to global ${OS_USERNAME}.
295    os_password  The password to be used to login to the OS.
296                 This defaults to global ${OS_PASSWORD}.
297    req_states   This is a list of states whose values are being requested by
298                 the caller.
299    os_up        If the caller knows that the os can't possibly be up, it can
300                 improve performance by passing os_up=False.  This function
301                 will then simply return default values for all requested os
302                 sub states.
303    quiet        Indicates whether status details (e.g. curl commands) should
304                 be written to the console.
305                 Defaults to either global value of ${QUIET} or to 1.
306    """
307
308    quiet = int(gp.get_var_value(quiet, 0))
309
310    # Set parm defaults where necessary and validate all parms.
311    if os_host == "":
312        os_host = BuiltIn().get_variable_value("${OS_HOST}")
313    error_message = gv.svalid_value(os_host, var_name="os_host",
314                                    invalid_values=[None, ""])
315    if error_message != "":
316        BuiltIn().fail(gp.sprint_error(error_message))
317
318    if os_username == "":
319        os_username = BuiltIn().get_variable_value("${OS_USERNAME}")
320    error_message = gv.svalid_value(os_username, var_name="os_username",
321                                    invalid_values=[None, ""])
322    if error_message != "":
323        BuiltIn().fail(gp.sprint_error(error_message))
324
325    if os_password == "":
326        os_password = BuiltIn().get_variable_value("${OS_PASSWORD}")
327    error_message = gv.svalid_value(os_password, var_name="os_password",
328                                    invalid_values=[None, ""])
329    if error_message != "":
330        BuiltIn().fail(gp.sprint_error(error_message))
331
332    invalid_req_states = [sub_state for sub_state in req_states
333                          if sub_state not in valid_os_req_states]
334    if len(invalid_req_states) > 0:
335        error_message = "The following req_states are not supported:\n" +\
336            gp.sprint_var(invalid_req_states)
337        BuiltIn().fail(gp.sprint_error(error_message))
338
339    # Initialize all substate values supported by this function.
340    os_ping = 0
341    os_login = 0
342    os_run_cmd = 0
343
344    if os_up:
345        if 'os_ping' in req_states:
346            # See if the OS pings.
347            rc, out_buf = gc.shell_cmd("ping -c 1 -w 2 " + os_host,
348                                       print_output=0, show_err=0,
349                                       ignore_err=1)
350            if rc == 0:
351                os_ping = 1
352
353        # Programming note: All attributes which do not require an ssh login
354        # should have been processed by this point.
355        master_req_login = ['os_login', 'os_run_cmd']
356        req_login = [sub_state for sub_state in req_states if sub_state in
357                     master_req_login]
358        must_login = (len(req_login) > 0)
359
360        if must_login:
361            output, stderr, rc = bsu.os_execute_command("uptime", quiet=quiet,
362                                                        ignore_err=1,
363                                                        time_out=20)
364            if rc == 0:
365                os_login = 1
366                os_run_cmd = 1
367            else:
368                gp.dprint_vars(output, stderr)
369                gp.dprint_vars(rc, 1)
370
371    os_state = DotDict()
372    for sub_state in req_states:
373        cmd_buf = "os_state['" + sub_state + "'] = str(" + sub_state + ")"
374        exec(cmd_buf)
375
376    return os_state
377
378
379def get_state(openbmc_host="",
380              openbmc_username="",
381              openbmc_password="",
382              os_host="",
383              os_username="",
384              os_password="",
385              req_states=default_req_states,
386              quiet=None):
387    r"""
388    Get component states such as chassis state, bmc state, etc, put them into a
389    dictionary and return them to the caller.
390
391    Note that all substate values are strings.
392
393    Description of arguments:
394    openbmc_host      The DNS name or IP address of the BMC.
395                      This defaults to global ${OPENBMC_HOST}.
396    openbmc_username  The username to be used to login to the BMC.
397                      This defaults to global ${OPENBMC_USERNAME}.
398    openbmc_password  The password to be used to login to the BMC.
399                      This defaults to global ${OPENBMC_PASSWORD}.
400    os_host           The DNS name or IP address of the operating system.
401                      This defaults to global ${OS_HOST}.
402    os_username       The username to be used to login to the OS.
403                      This defaults to global ${OS_USERNAME}.
404    os_password       The password to be used to login to the OS.
405                      This defaults to global ${OS_PASSWORD}.
406    req_states        This is a list of states whose values are being requested
407                      by the caller.
408    quiet             Indicates whether status details (e.g. curl commands)
409                      should be written to the console.
410                      Defaults to either global value of ${QUIET} or to 1.
411    """
412
413    quiet = int(gp.get_var_value(quiet, 0))
414
415    # Set parm defaults where necessary and validate all parms.
416    if openbmc_host == "":
417        openbmc_host = BuiltIn().get_variable_value("${OPENBMC_HOST}")
418    error_message = gv.svalid_value(openbmc_host,
419                                    var_name="openbmc_host",
420                                    invalid_values=[None, ""])
421    if error_message != "":
422        BuiltIn().fail(gp.sprint_error(error_message))
423
424    if openbmc_username == "":
425        openbmc_username = BuiltIn().get_variable_value("${OPENBMC_USERNAME}")
426    error_message = gv.svalid_value(openbmc_username,
427                                    var_name="openbmc_username",
428                                    invalid_values=[None, ""])
429    if error_message != "":
430        BuiltIn().fail(gp.sprint_error(error_message))
431
432    if openbmc_password == "":
433        openbmc_password = BuiltIn().get_variable_value("${OPENBMC_PASSWORD}")
434    error_message = gv.svalid_value(openbmc_password,
435                                    var_name="openbmc_password",
436                                    invalid_values=[None, ""])
437    if error_message != "":
438        BuiltIn().fail(gp.sprint_error(error_message))
439
440    # NOTE: OS parms are optional.
441    if os_host == "":
442        os_host = BuiltIn().get_variable_value("${OS_HOST}")
443        if os_host is None:
444            os_host = ""
445
446    if os_username is "":
447        os_username = BuiltIn().get_variable_value("${OS_USERNAME}")
448        if os_username is None:
449            os_username = ""
450
451    if os_password is "":
452        os_password = BuiltIn().get_variable_value("${OS_PASSWORD}")
453        if os_password is None:
454            os_password = ""
455
456    invalid_req_states = [sub_state for sub_state in req_states
457                          if sub_state not in valid_req_states]
458    if len(invalid_req_states) > 0:
459        error_message = "The following req_states are not supported:\n" +\
460            gp.sprint_var(invalid_req_states)
461        BuiltIn().fail(gp.sprint_error(error_message))
462
463    # Initialize all substate values supported by this function.
464    ping = 0
465    packet_loss = ''
466    uptime = ''
467    epoch_seconds = ''
468    rest = ''
469    chassis = ''
470    requested_chassis = ''
471    bmc = ''
472    requested_bmc = ''
473    boot_progress = ''
474    operating_system = ''
475    host = ''
476    requested_host = ''
477    attempts_left = ''
478
479    # Get the component states.
480    if 'ping' in req_states:
481        # See if the OS pings.
482        rc, out_buf = gc.shell_cmd("ping -c 1 -w 2 " + openbmc_host,
483                                   print_output=0, show_err=0,
484                                   ignore_err=1)
485        if rc == 0:
486            ping = 1
487
488    if 'packet_loss' in req_states:
489        # See if the OS pings.
490        cmd_buf = "ping -c 5 -w 5 " + openbmc_host +\
491            " | egrep 'packet loss' | sed -re 's/.* ([0-9]+)%.*/\\1/g'"
492        rc, out_buf = gc.shell_cmd(cmd_buf,
493                                   print_output=0, show_err=0,
494                                   ignore_err=1)
495        if rc == 0:
496            packet_loss = out_buf.rstrip("\n")
497
498    if 'uptime' in req_states:
499        # Sometimes reading uptime results in a blank value. Call with
500        # wait_until_keyword_succeeds to ensure a non-blank value is obtained.
501        remote_cmd_buf = "read uptime filler 2>/dev/null < /proc/uptime" +\
502            " && [ ! -z \"${uptime}\" ] && echo ${uptime}"
503        cmd_buf = ["BMC Execute Command",
504                   re.sub('\\$', '\\$', remote_cmd_buf), 'quiet=1',
505                   'test_mode=0']
506        gp.print_issuing(cmd_buf, 0)
507        gp.print_issuing(remote_cmd_buf, 0)
508        try:
509            stdout, stderr, rc =\
510                BuiltIn().wait_until_keyword_succeeds("10 sec", "0 sec",
511                                                      *cmd_buf)
512            if rc == 0 and stderr == "":
513                uptime = stdout
514        except AssertionError as my_assertion_error:
515            pass
516
517    if 'epoch_seconds' in req_states:
518        date_cmd_buf = "date -u +%s"
519        if USE_BMC_EPOCH_TIME:
520            cmd_buf = ["BMC Execute Command", date_cmd_buf, 'quiet=${1}']
521            if not quiet:
522                gp.print_issuing(cmd_buf)
523            status, ret_values = \
524                BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
525            if status == "PASS":
526                stdout, stderr, rc = ret_values
527                if rc == 0 and stderr == "":
528                    epoch_seconds = stdout.rstrip("\n")
529        else:
530            shell_rc, out_buf = gc.cmd_fnc_u(date_cmd_buf,
531                                             quiet=quiet,
532                                             print_output=0)
533            if shell_rc == 0:
534                epoch_seconds = out_buf.rstrip("\n")
535
536    master_req_rest = ['rest', 'host', 'requested_host', 'operating_system',
537                       'attempts_left', 'boot_progress', 'chassis',
538                       'requested_chassis' 'bmc' 'requested_bmc']
539
540    req_rest = [sub_state for sub_state in req_states if sub_state in
541                master_req_rest]
542    need_rest = (len(req_rest) > 0)
543    state = DotDict()
544    if need_rest:
545        cmd_buf = ["Read Properties", SYSTEM_STATE_URI + "enumerate",
546                   "quiet=${" + str(quiet) + "}"]
547        gp.dprint_issuing(cmd_buf)
548        status, ret_values = \
549            BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
550        if status == "PASS":
551            state['rest'] = '1'
552        else:
553            state['rest'] = '0'
554
555        if int(state['rest']):
556            for url_path in ret_values:
557                for attr_name in ret_values[url_path]:
558                    # Create a state key value based on the attr_name.
559                    try:
560                        ret_values[url_path][attr_name] = \
561                            re.sub(r'.*\.', "",
562                                   ret_values[url_path][attr_name])
563                    except TypeError:
564                        pass
565                    # Do some key name manipulations.
566                    new_attr_name = re.sub(r'^Current|(State|Transition)$',
567                                           "", attr_name)
568                    new_attr_name = re.sub(r'BMC', r'Bmc', new_attr_name)
569                    new_attr_name = re.sub(r'([A-Z][a-z])', r'_\1',
570                                           new_attr_name)
571                    new_attr_name = new_attr_name.lower().lstrip("_")
572                    new_attr_name = re.sub(r'power', r'chassis', new_attr_name)
573                    if new_attr_name in req_states:
574                        state[new_attr_name] = ret_values[url_path][attr_name]
575
576    for sub_state in req_states:
577        if sub_state in state:
578            continue
579        if sub_state.startswith("os_"):
580            # We pass "os_" requests on to get_os_state.
581            continue
582        cmd_buf = "state['" + sub_state + "'] = str(" + sub_state + ")"
583        exec(cmd_buf)
584
585    if os_host == "":
586        # The caller has not specified an os_host so as far as we're concerned,
587        # it doesn't exist.
588        return state
589
590    os_req_states = [sub_state for sub_state in req_states
591                     if sub_state.startswith('os_')]
592
593    if len(os_req_states) > 0:
594        # The caller has specified an os_host and they have requested
595        # information on os substates.
596
597        # Based on the information gathered on bmc, we'll try to make a
598        # determination of whether the os is even up.  We'll pass the result
599        # of that assessment to get_os_state to enhance performance.
600        os_up_match = DotDict()
601        for sub_state in master_os_up_match:
602            if sub_state in req_states:
603                os_up_match[sub_state] = master_os_up_match[sub_state]
604        os_up = compare_states(state, os_up_match)
605        os_state = get_os_state(os_host=os_host,
606                                os_username=os_username,
607                                os_password=os_password,
608                                req_states=os_req_states,
609                                os_up=os_up,
610                                quiet=quiet)
611        # Append os_state dictionary to ours.
612        state.update(os_state)
613
614    return state
615
616
617exit_wait_early_message = ""
618
619
620def set_exit_wait_early_message(value):
621    r"""
622    Set global exit_wait_early_message to the indicated value.
623
624    This is a mechanism by which the programmer can do an early exit from
625    wait_until_keyword_succeeds() based on some special condition.
626
627    Description of argument(s):
628    value                           The value to assign to the global
629                                    exit_wait_early_message.
630    """
631
632    global exit_wait_early_message
633    exit_wait_early_message = value
634
635
636def check_state(match_state,
637                invert=0,
638                print_string="",
639                openbmc_host="",
640                openbmc_username="",
641                openbmc_password="",
642                os_host="",
643                os_username="",
644                os_password="",
645                quiet=None):
646    r"""
647    Check that the Open BMC machine's composite state matches the specified
648    state.  On success, this keyword returns the machine's composite state as a
649    dictionary.
650
651    Description of arguments:
652    match_state       A dictionary whose key/value pairs are "state field"/
653                      "state value".  The state value is interpreted as a
654                      regular expression.  Example call from robot:
655                      ${match_state}=  Create Dictionary  chassis=^On$
656                      ...  bmc=^Ready$
657                      ...  boot_progress=^OSStart$
658                      ${state}=  Check State  &{match_state}
659    invert            If this flag is set, this function will succeed if the
660                      states do NOT match.
661    print_string      This function will print this string to the console prior
662                      to getting the state.
663    openbmc_host      The DNS name or IP address of the BMC.
664                      This defaults to global ${OPENBMC_HOST}.
665    openbmc_username  The username to be used to login to the BMC.
666                      This defaults to global ${OPENBMC_USERNAME}.
667    openbmc_password  The password to be used to login to the BMC.
668                      This defaults to global ${OPENBMC_PASSWORD}.
669    os_host           The DNS name or IP address of the operating system.
670                      This defaults to global ${OS_HOST}.
671    os_username       The username to be used to login to the OS.
672                      This defaults to global ${OS_USERNAME}.
673    os_password       The password to be used to login to the OS.
674                      This defaults to global ${OS_PASSWORD}.
675    quiet             Indicates whether status details should be written to the
676                      console.  Defaults to either global value of ${QUIET} or
677                      to 1.
678    """
679
680    quiet = int(gp.get_var_value(quiet, 0))
681
682    gp.gp_print(print_string)
683
684    try:
685        match_state = return_state_constant(match_state)
686    except TypeError:
687        pass
688
689    req_states = match_state.keys()
690    # Initialize state.
691    state = get_state(openbmc_host=openbmc_host,
692                      openbmc_username=openbmc_username,
693                      openbmc_password=openbmc_password,
694                      os_host=os_host,
695                      os_username=os_username,
696                      os_password=os_password,
697                      req_states=req_states,
698                      quiet=quiet)
699    if not quiet:
700        gp.print_var(state)
701
702    if exit_wait_early_message != "":
703        # The exit_wait_early_message has been set by a signal handler so we
704        # will exit "successfully".  It is incumbent upon the calling function
705        # (e.g. wait_state) to check/clear this variable and to fail
706        # appropriately.
707        return state
708
709    match = compare_states(state, match_state)
710
711    if invert and match:
712        fail_msg = "The current state of the machine matches the match" +\
713                   " state:\n" + gp.sprint_varx("state", state)
714        BuiltIn().fail("\n" + gp.sprint_error(fail_msg))
715    elif not invert and not match:
716        fail_msg = "The current state of the machine does NOT match the" +\
717                   " match state:\n" +\
718                   gp.sprint_varx("state", state)
719        BuiltIn().fail("\n" + gp.sprint_error(fail_msg))
720
721    return state
722
723
724def wait_state(match_state=(),
725               wait_time="1 min",
726               interval="1 second",
727               invert=0,
728               openbmc_host="",
729               openbmc_username="",
730               openbmc_password="",
731               os_host="",
732               os_username="",
733               os_password="",
734               quiet=None):
735    r"""
736    Wait for the Open BMC machine's composite state to match the specified
737    state.  On success, this keyword returns the machine's composite state as
738    a dictionary.
739
740    Description of arguments:
741    match_state       A dictionary whose key/value pairs are "state field"/
742                      "state value".  See check_state (above) for details.
743                      This value may also be any string accepted by
744                      return_state_constant (e.g. "standby_match_state").
745                      In such a case this function will call
746                      return_state_constant to convert it to a proper
747                      dictionary as described above.
748    wait_time         The total amount of time to wait for the desired state.
749                      This value may be expressed in Robot Framework's time
750                      format (e.g. 1 minute, 2 min 3 s, 4.5).
751    interval          The amount of time between state checks.
752                      This value may be expressed in Robot Framework's time
753                      format (e.g. 1 minute, 2 min 3 s, 4.5).
754    invert            If this flag is set, this function will for the state of
755                      the machine to cease to match the match state.
756    openbmc_host      The DNS name or IP address of the BMC.
757                      This defaults to global ${OPENBMC_HOST}.
758    openbmc_username  The username to be used to login to the BMC.
759                      This defaults to global ${OPENBMC_USERNAME}.
760    openbmc_password  The password to be used to login to the BMC.
761                      This defaults to global ${OPENBMC_PASSWORD}.
762    os_host           The DNS name or IP address of the operating system.
763                      This defaults to global ${OS_HOST}.
764    os_username       The username to be used to login to the OS.
765                      This defaults to global ${OS_USERNAME}.
766    os_password       The password to be used to login to the OS.
767                      This defaults to global ${OS_PASSWORD}.
768    quiet             Indicates whether status details should be written to the
769                      console.  Defaults to either global value of ${QUIET} or
770                      to 1.
771    """
772
773    quiet = int(gp.get_var_value(quiet, 0))
774
775    try:
776        match_state = return_state_constant(match_state)
777    except TypeError:
778        pass
779
780    if not quiet:
781        if invert:
782            alt_text = "cease to "
783        else:
784            alt_text = ""
785        gp.print_timen("Checking every " + str(interval) + " for up to "
786                       + str(wait_time) + " for the state of the machine to "
787                       + alt_text + "match the state shown below.")
788        gp.print_var(match_state)
789
790    if quiet:
791        print_string = ""
792    else:
793        print_string = "#"
794
795    debug = int(BuiltIn().get_variable_value("${debug}", "0"))
796    if debug:
797        # In debug we print state so no need to print the "#".
798        print_string = ""
799    check_state_quiet = 1 - debug
800    cmd_buf = ["Check State", match_state, "invert=${" + str(invert) + "}",
801               "print_string=" + print_string, "openbmc_host=" + openbmc_host,
802               "openbmc_username=" + openbmc_username,
803               "openbmc_password=" + openbmc_password, "os_host=" + os_host,
804               "os_username=" + os_username, "os_password=" + os_password,
805               "quiet=${" + str(check_state_quiet) + "}"]
806    gp.dprint_issuing(cmd_buf)
807    try:
808        state = BuiltIn().wait_until_keyword_succeeds(wait_time, interval,
809                                                      *cmd_buf)
810    except AssertionError as my_assertion_error:
811        gp.printn()
812        message = my_assertion_error.args[0]
813        BuiltIn().fail(message)
814
815    if exit_wait_early_message:
816        # The global exit_wait_early_message was set by a signal handler
817        # indicating that we should fail.
818        message = exit_wait_early_message
819        # Clear the exit_wait_early_message variable for future use.
820        set_exit_wait_early_message("")
821        BuiltIn().fail(gp.sprint_error(message))
822
823    if not quiet:
824        gp.printn()
825        if invert:
826            gp.print_timen("The states no longer match:")
827        else:
828            gp.print_timen("The states match:")
829        gp.print_var(state)
830
831    return state
832
833
834def wait_for_comm_cycle(start_boot_seconds,
835                        quiet=None):
836    r"""
837    Wait for communications to the BMC to stop working and then resume working.
838    This function is useful when you have initiated some kind of reboot.
839
840    Description of arguments:
841    start_boot_seconds  The time that the boot test started.  The format is the
842                        epoch time in seconds, i.e. the number of seconds since
843                        1970-01-01 00:00:00 UTC.  This value should be obtained
844                        from the BMC so that it is not dependent on any kind of
845                        synchronization between this machine and the target BMC
846                        This will allow this program to work correctly even in
847                        a simulated environment.  This value should be obtained
848                        by the caller prior to initiating a reboot.  It can be
849                        obtained as follows:
850                        state = st.get_state(req_states=['epoch_seconds'])
851    """
852
853    quiet = int(gp.get_var_value(quiet, 0))
854
855    # Validate parms.
856    error_message = gv.svalid_integer(start_boot_seconds,
857                                      var_name="start_boot_seconds")
858    if error_message != "":
859        BuiltIn().fail(gp.sprint_error(error_message))
860
861    match_state = anchor_state(DotDict([('packet_loss', '100')]))
862    # Wait for 100% packet loss trying to ping machine.
863    wait_state(match_state, wait_time="8 mins", interval="0 seconds")
864
865    match_state['packet_loss'] = '^0$'
866    # Wait for 0% packet loss trying to ping machine.
867    wait_state(match_state, wait_time="8 mins", interval="0 seconds")
868
869    # Get the uptime and epoch seconds for comparisons.  We want to be sure
870    # that the uptime is less than the elapsed boot time.  Further proof that
871    # a reboot has indeed occurred (vs random network instability giving a
872    # false positive.  We also use wait_state because the BMC may take a short
873    # while to be ready to process SSH requests.
874    match_state = DotDict([('uptime', '^[0-9\\.]+$'),
875                           ('epoch_seconds', '^[0-9]+$')])
876    state = wait_state(match_state, wait_time="2 mins", interval="1 second")
877
878    elapsed_boot_time = int(state['epoch_seconds']) - start_boot_seconds
879    gp.qprint_var(elapsed_boot_time)
880    if state['uptime'] == "":
881        error_message = "Unable to obtain uptime from the BMC. BMC is not" +\
882            " communicating."
883        BuiltIn().fail(gp.sprint_error(error_message))
884    if int(float(state['uptime'])) < elapsed_boot_time:
885        uptime = state['uptime']
886        gp.qprint_var(uptime)
887        gp.qprint_timen("The uptime is less than the elapsed boot time,"
888                        + " as expected.")
889    else:
890        error_message = "The uptime is greater than the elapsed boot time," +\
891                        " which is unexpected:\n" +\
892                        gp.sprint_var(start_boot_seconds) +\
893                        gp.sprint_var(state)
894        BuiltIn().fail(gp.sprint_error(error_message))
895
896    gp.qprint_timen("Verifying that REST API interface is working.")
897    match_state = DotDict([('rest', '^1$')])
898    state = wait_state(match_state, wait_time="5 mins", interval="2 seconds")
899