xref: /openbmc/openbmc-test-automation/lib/state.py (revision 95686eff54a2d9341e5c1e52e15f949b2ed7e5bc)
1#!/usr/bin/env python
2
3r"""
4This module contains functions having to do with machine state: get_state,
5check_state, wait_state, etc.
6
7The 'State' is a composite of many pieces of data.  Therefore, the functions
8in this module define state as an ordered dictionary.  Here is an example of
9some test output showing machine state:
10
11default_state:
12  default_state[chassis]:                         On
13  default_state[boot_progress]:                   OSStart
14  default_state[operating_system]:                BootComplete
15  default_state[host]:                            Running
16  default_state[os_ping]:                         1
17  default_state[os_login]:                        1
18  default_state[os_run_cmd]:                      1
19
20Different users may very well have different needs when inquiring about
21state.  Support for new pieces of state information may be added to this
22module as needed.
23
24By using the wait_state function, a caller can start a boot and then wait for
25a precisely defined state to indicate that the boot has succeeded.  If
26the boot fails, they can see exactly why by looking at the current state as
27compared with the expected state.
28"""
29
30import gen_print as gp
31import gen_robot_print as grp
32import gen_valid as gv
33import gen_robot_utils as gru
34import gen_cmd as gc
35
36import commands
37from robot.libraries.BuiltIn import BuiltIn
38from robot.utils import DotDict
39
40import re
41import os
42import sys
43import imp
44
45
46# We need utils.robot to get keywords like "Get Chassis Power State".
47gru.my_import_resource("utils.robot")
48gru.my_import_resource("state_manager.robot")
49
50base_path = os.path.dirname(os.path.dirname(
51                            imp.find_module("gen_robot_print")[1])) + os.sep
52sys.path.append(base_path + "data/")
53
54# Previously, I had this coded:
55# import variables as var
56# However, we ran into a problem where a robot program did this...
57# Variables           ../../lib/ras/variables.py
58# Prior to doing this...
59# Library            ../lib/state.py
60
61# This caused the wrong variables.py file to be selected.  Attempts to fix this
62# have failed so far.  For the moment, we will hard-code the value we need from
63# the file.
64
65SYSTEM_STATE_URI = "/xyz/openbmc_project/state/"
66
67# The BMC code has recently been changed as far as what states are defined and
68# what the state values can be.  This module now has a means of processing both
69# the old style state (i.e. OBMC_STATES_VERSION = 0) and the new style (i.e.
70# OBMC_STATES_VERSION = 1).
71# The caller can set environment variable OBMC_STATES_VERSION to dictate
72# whether we're processing old or new style states.  If OBMC_STATES_VERSION is
73# not set it will default to 1.
74
75# As of the present moment, OBMC_STATES_VERSION of 0 is for cold that is so old
76# that it is no longer worthwhile to maintain.  The OBMC_STATES_VERSION 0 code
77# is being removed but the OBMC_STATES_VERSION value will stay for now in the
78# event that it is needed in the future.
79
80OBMC_STATES_VERSION = int(os.environ.get('OBMC_STATES_VERSION', 1))
81
82# When a user calls get_state w/o specifying req_states, default_req_states
83# is used as its value.
84default_req_states = ['rest',
85                      'chassis',
86                      'bmc',
87                      'boot_progress',
88                      'operating_system',
89                      'host',
90                      'os_ping',
91                      'os_login',
92                      'os_run_cmd']
93
94# valid_req_states is a list of sub states supported by the get_state function.
95# valid_req_states, default_req_states and master_os_up_match are used by the
96# get_state function.
97valid_req_states = ['ping',
98                    'packet_loss',
99                    'uptime',
100                    'epoch_seconds',
101                    'rest',
102                    'chassis',
103                    'requested_chassis',
104                    'bmc',
105                    'requested_bmc',
106                    'boot_progress',
107                    'operating_system',
108                    'host',
109                    'requested_host',
110                    'attempts_left',
111                    'os_ping',
112                    'os_login',
113                    'os_run_cmd']
114
115# valid_os_req_states and default_os_req_states are used by the os_get_state
116# function.
117# valid_os_req_states is a list of state information supported by the
118# get_os_state function.
119valid_os_req_states = ['os_ping',
120                       'os_login',
121                       'os_run_cmd']
122# When a user calls get_os_state w/o specifying req_states,
123# default_os_req_states is used as its value.
124default_os_req_states = ['os_ping',
125                         'os_login',
126                         'os_run_cmd']
127
128# Presently, some BMCs appear to not keep time very well.  This environment
129# variable directs the get_state function to use either the BMC's epoch time
130# or the local epoch time.
131USE_BMC_EPOCH_TIME = int(os.environ.get('USE_BMC_EPOCH_TIME', 0))
132
133# Useful state constant definition(s).
134# A match state for checking that the system is at "standby".
135standby_match_state = DotDict([('rest', '^1$'),
136                               ('chassis', '^Off$'),
137                               ('bmc', '^Ready$'),
138                               ('boot_progress', ''),
139                               ('operating_system', ''),
140                               ('host', '')])
141
142# default_state is an initial value which may be of use to callers.
143default_state = DotDict([('rest', '1'),
144                         ('chassis', 'On'),
145                         ('bmc', 'Ready'),
146                         ('boot_progress', 'OSStart'),
147                         ('operating_system', 'BootComplete'),
148                         ('host', 'Running'),
149                         ('os_ping', '1'),
150                         ('os_login', '1'),
151                         ('os_run_cmd', '1')])
152
153# A master dictionary to determine whether the os may be up.
154master_os_up_match = DotDict([('chassis', '^On$'),
155                              ('bmc', '^Ready$'),
156                              ('boot_progress',
157                               'FW Progress, Starting OS|OSStart'),
158                              ('operating_system', 'BootComplete'),
159                              ('host', '^Running$')])
160
161invalid_state_match = DotDict([('rest', '^$'),
162                               ('chassis', '^$'),
163                               ('bmc', '^$'),
164                               ('boot_progress', '^$'),
165                               ('operating_system', '^$'),
166                               ('host', '^$')])
167
168
169###############################################################################
170def return_default_state():
171
172    r"""
173    Return default state dictionary.
174
175    default_state is an initial value which may be of use to callers.
176    """
177
178    return default_state
179
180###############################################################################
181
182
183valid_state_constants = ['default', 'standby_match_state']
184
185
186###############################################################################
187def return_state_constant(state_name='default'):
188
189    r"""
190    Return default state dictionary.
191
192    default_state is an initial value which may be of use to callers.
193    """
194
195    error_message = gv.svalid_value(state_name, var_name='state_name',
196                                    valid_values=valid_state_constants)
197    if error_message != "":
198        BuiltIn().fail(gp.sprint_error(error_message))
199
200    if state_name == 'default':
201        return default_state
202    elif state_name == 'standby_match_state':
203        return standby_match_state
204
205###############################################################################
206
207
208###############################################################################
209def anchor_state(state):
210
211    r"""
212    Add regular expression anchors ("^" and "$") to the beginning and end of
213    each item in the state dictionary passed in.  Return the resulting
214    dictionary.
215
216    Description of Arguments:
217    state    A dictionary such as the one returned by the get_state()
218             function.
219    """
220
221    anchored_state = state.copy()
222    for key, match_state_value in anchored_state.items():
223        anchored_state[key] = "^" + str(anchored_state[key]) + "$"
224
225    return anchored_state
226
227###############################################################################
228
229
230###############################################################################
231def strip_anchor_state(state):
232
233    r"""
234    Strip regular expression anchors ("^" and "$") from the beginning and end
235    of each item in the state dictionary passed in.  Return the resulting
236    dictionary.
237
238    Description of Arguments:
239    state    A dictionary such as the one returned by the get_state()
240             function.
241    """
242
243    stripped_state = state.copy()
244    for key, match_state_value in stripped_state.items():
245        stripped_state[key] = stripped_state[key].strip("^$")
246
247    return stripped_state
248
249###############################################################################
250
251
252###############################################################################
253def compare_states(state,
254                   match_state,
255                   match_type='and'):
256
257    r"""
258    Compare 2 state dictionaries.  Return True if they match and False if they
259    don't.  Note that the match_state dictionary does not need to have an entry
260    corresponding to each entry in the state dictionary.  But for each entry
261    that it does have, the corresponding state entry will be checked for a
262    match.
263
264    Description of arguments:
265    state           A state dictionary such as the one returned by the
266                    get_state function.
267    match_state     A dictionary whose key/value pairs are "state field"/
268                    "state value".  The state value is interpreted as a
269                    regular expression.  Every value in this dictionary is
270                    considered.  When match_type is 'and', if each and every
271                    comparison matches, the two dictionaries are considered to
272                    be matching.  If match_type is 'or', if any two of the
273                    elements compared match, the two dictionaries are
274                    considered to be matching.
275    match_type      This may be 'and' or 'or'.
276    """
277
278    error_message = gv.svalid_value(match_type, var_name="match_type",
279                                    valid_values=['and', 'or'])
280    if error_message != "":
281        BuiltIn().fail(gp.sprint_error(error_message))
282
283    default_match = (match_type == 'and')
284    for key, match_state_value in match_state.items():
285        # Blank match_state_value means "don't care".
286        if match_state_value == "":
287            continue
288        try:
289            match = (re.match(match_state_value, str(state[key])) is not None)
290        except KeyError:
291            match = False
292
293        if match != default_match:
294            return match
295
296    return default_match
297
298###############################################################################
299
300
301###############################################################################
302def get_os_state(os_host="",
303                 os_username="",
304                 os_password="",
305                 req_states=default_os_req_states,
306                 os_up=True,
307                 quiet=None):
308
309    r"""
310    Get component states for the operating system such as ping, login,
311    etc, put them into a dictionary and return them to the caller.
312
313    Note that all substate values are strings.
314
315    Description of arguments:
316    os_host      The DNS name or IP address of the operating system.
317                 This defaults to global ${OS_HOST}.
318    os_username  The username to be used to login to the OS.
319                 This defaults to global ${OS_USERNAME}.
320    os_password  The password to be used to login to the OS.
321                 This defaults to global ${OS_PASSWORD}.
322    req_states   This is a list of states whose values are being requested by
323                 the caller.
324    os_up        If the caller knows that the os can't possibly be up, it can
325                 improve performance by passing os_up=False.  This function
326                 will then simply return default values for all requested os
327                 sub states.
328    quiet        Indicates whether status details (e.g. curl commands) should
329                 be written to the console.
330                 Defaults to either global value of ${QUIET} or to 1.
331    """
332
333    quiet = int(gp.get_var_value(quiet, 0))
334
335    # Set parm defaults where necessary and validate all parms.
336    if os_host == "":
337        os_host = BuiltIn().get_variable_value("${OS_HOST}")
338    error_message = gv.svalid_value(os_host, var_name="os_host",
339                                    invalid_values=[None, ""])
340    if error_message != "":
341        BuiltIn().fail(gp.sprint_error(error_message))
342
343    if os_username == "":
344        os_username = BuiltIn().get_variable_value("${OS_USERNAME}")
345    error_message = gv.svalid_value(os_username, var_name="os_username",
346                                    invalid_values=[None, ""])
347    if error_message != "":
348        BuiltIn().fail(gp.sprint_error(error_message))
349
350    if os_password == "":
351        os_password = BuiltIn().get_variable_value("${OS_PASSWORD}")
352    error_message = gv.svalid_value(os_password, var_name="os_password",
353                                    invalid_values=[None, ""])
354    if error_message != "":
355        BuiltIn().fail(gp.sprint_error(error_message))
356
357    invalid_req_states = [sub_state for sub_state in req_states
358                          if sub_state not in valid_os_req_states]
359    if len(invalid_req_states) > 0:
360        error_message = "The following req_states are not supported:\n" +\
361            gp.sprint_var(invalid_req_states)
362        BuiltIn().fail(gp.sprint_error(error_message))
363
364    # Initialize all substate values supported by this function.
365    os_ping = 0
366    os_login = 0
367    os_run_cmd = 0
368
369    if os_up:
370        if 'os_ping' in req_states:
371            # See if the OS pings.
372            cmd_buf = "ping -c 1 -w 2 " + os_host
373            if not quiet:
374                gp.pissuing(cmd_buf)
375            rc, out_buf = commands.getstatusoutput(cmd_buf)
376            if rc == 0:
377                os_ping = 1
378
379        # Programming note: All attributes which do not require an ssh login
380        # should have been processed by this point.
381        master_req_login = ['os_login', 'os_run_cmd']
382        req_login = [sub_state for sub_state in req_states if sub_state in
383                     master_req_login]
384        must_login = (len(req_login) > 0)
385
386        if must_login:
387            # Open SSH connection to OS.  Note that this doesn't fail even when
388            # the OS is not up.
389            cmd_buf = ["SSHLibrary.Open Connection", os_host]
390            if not quiet:
391                grp.rpissuing_keyword(cmd_buf)
392            ix = BuiltIn().run_keyword(*cmd_buf)
393
394            # Login to OS.
395            cmd_buf = ["Login", os_username, os_password]
396            if not quiet:
397                grp.rpissuing_keyword(cmd_buf)
398            status, ret_values = \
399                BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
400            if status == "PASS":
401                os_login = 1
402            else:
403                gp.dprint_var(status)
404                gp.dprint_var(ret_values)
405
406            if os_login:
407                if 'os_run_cmd' in req_states:
408                    # Try running a simple command (uptime) on the OS.
409                    cmd_buf = ["Execute Command", "uptime",
410                               "return_stderr=True", "return_rc=True"]
411                    if not quiet:
412                        grp.rpissuing_keyword(cmd_buf)
413                    # Note that in spite of its name, there are occasions
414                    # where run_keyword_and_ignore_error can fail.
415                    status, ret_values = \
416                        BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
417                    if status == "PASS":
418                        stdout, stderr, rc = ret_values
419                        if rc == 0 and stderr == "":
420                            os_run_cmd = 1
421                        else:
422                            gp.dprint_var(status)
423                            gp.dprint_var(stdout)
424                            gp.dprint_var(stderr)
425                            gp.dprint_var(rc)
426                    else:
427                        gp.dprint_var(status)
428                        gp.dprint_var(ret_values)
429
430    os_state = DotDict()
431    for sub_state in req_states:
432        cmd_buf = "os_state['" + sub_state + "'] = str(" + sub_state + ")"
433        exec(cmd_buf)
434
435    return os_state
436
437###############################################################################
438
439
440###############################################################################
441def get_state(openbmc_host="",
442              openbmc_username="",
443              openbmc_password="",
444              os_host="",
445              os_username="",
446              os_password="",
447              req_states=default_req_states,
448              quiet=None):
449
450    r"""
451    Get component states such as chassis state, bmc state, etc, put them into a
452    dictionary and return them to the caller.
453
454    Note that all substate values are strings.
455
456    Description of arguments:
457    openbmc_host      The DNS name or IP address of the BMC.
458                      This defaults to global ${OPENBMC_HOST}.
459    openbmc_username  The username to be used to login to the BMC.
460                      This defaults to global ${OPENBMC_USERNAME}.
461    openbmc_password  The password to be used to login to the BMC.
462                      This defaults to global ${OPENBMC_PASSWORD}.
463    os_host           The DNS name or IP address of the operating system.
464                      This defaults to global ${OS_HOST}.
465    os_username       The username to be used to login to the OS.
466                      This defaults to global ${OS_USERNAME}.
467    os_password       The password to be used to login to the OS.
468                      This defaults to global ${OS_PASSWORD}.
469    req_states        This is a list of states whose values are being requested
470                      by the caller.
471    quiet             Indicates whether status details (e.g. curl commands)
472                      should be written to the console.
473                      Defaults to either global value of ${QUIET} or to 1.
474    """
475
476    quiet = int(gp.get_var_value(quiet, 0))
477
478    # Set parm defaults where necessary and validate all parms.
479    if openbmc_host == "":
480        openbmc_host = BuiltIn().get_variable_value("${OPENBMC_HOST}")
481    error_message = gv.svalid_value(openbmc_host,
482                                    var_name="openbmc_host",
483                                    invalid_values=[None, ""])
484    if error_message != "":
485        BuiltIn().fail(gp.sprint_error(error_message))
486
487    if openbmc_username == "":
488        openbmc_username = BuiltIn().get_variable_value("${OPENBMC_USERNAME}")
489    error_message = gv.svalid_value(openbmc_username,
490                                    var_name="openbmc_username",
491                                    invalid_values=[None, ""])
492    if error_message != "":
493        BuiltIn().fail(gp.sprint_error(error_message))
494
495    if openbmc_password == "":
496        openbmc_password = BuiltIn().get_variable_value("${OPENBMC_PASSWORD}")
497    error_message = gv.svalid_value(openbmc_password,
498                                    var_name="openbmc_password",
499                                    invalid_values=[None, ""])
500    if error_message != "":
501        BuiltIn().fail(gp.sprint_error(error_message))
502
503    # NOTE: OS parms are optional.
504    if os_host == "":
505        os_host = BuiltIn().get_variable_value("${OS_HOST}")
506        if os_host is None:
507            os_host = ""
508
509    if os_username is "":
510        os_username = BuiltIn().get_variable_value("${OS_USERNAME}")
511        if os_username is None:
512            os_username = ""
513
514    if os_password is "":
515        os_password = BuiltIn().get_variable_value("${OS_PASSWORD}")
516        if os_password is None:
517            os_password = ""
518
519    invalid_req_states = [sub_state for sub_state in req_states
520                          if sub_state not in valid_req_states]
521    if len(invalid_req_states) > 0:
522        error_message = "The following req_states are not supported:\n" +\
523            gp.sprint_var(invalid_req_states)
524        BuiltIn().fail(gp.sprint_error(error_message))
525
526    # Initialize all substate values supported by this function.
527    ping = 0
528    packet_loss = ''
529    uptime = ''
530    epoch_seconds = ''
531    rest = ''
532    chassis = ''
533    requested_chassis = ''
534    bmc = ''
535    requested_bmc = ''
536    boot_progress = ''
537    operating_system = ''
538    host = ''
539    requested_host = ''
540    attempts_left = ''
541
542    # Get the component states.
543    if 'ping' in req_states:
544        # See if the OS pings.
545        cmd_buf = "ping -c 1 -w 2 " + openbmc_host
546        if not quiet:
547            gp.pissuing(cmd_buf)
548        rc, out_buf = commands.getstatusoutput(cmd_buf)
549        if rc == 0:
550            ping = 1
551
552    if 'packet_loss' in req_states:
553        # See if the OS pings.
554        cmd_buf = "ping -c 5 -w 5 " + openbmc_host +\
555            " | egrep 'packet loss' | sed -re 's/.* ([0-9]+)%.*/\\1/g'"
556        if not quiet:
557            gp.pissuing(cmd_buf)
558        rc, out_buf = commands.getstatusoutput(cmd_buf)
559        if rc == 0:
560            packet_loss = out_buf.rstrip("\n")
561
562    if 'uptime' in req_states:
563        # Sometimes reading uptime results in a blank value. Call with
564        # wait_until_keyword_succeeds to ensure a non-blank value is obtained.
565        remote_cmd_buf = "read uptime filler 2>/dev/null < /proc/uptime" +\
566            " && [ ! -z \"${uptime}\" ] && echo ${uptime}"
567        cmd_buf = ["BMC Execute Command", re.sub(r'\$', '\$', remote_cmd_buf),
568                   'quiet=1']
569        if not quiet:
570            grp.rpissuing_keyword(cmd_buf)
571            grp.rpissuing(remote_cmd_buf)
572        try:
573            stdout, stderr, rc =\
574                BuiltIn().wait_until_keyword_succeeds("5 sec", "0 sec",
575                                                      *cmd_buf)
576            if rc == 0 and stderr == "":
577                uptime = stdout
578        except AssertionError as my_assertion_error:
579            pass
580
581    if 'epoch_seconds' in req_states:
582        date_cmd_buf = "date -u +%s"
583        if USE_BMC_EPOCH_TIME:
584            cmd_buf = ["BMC Execute Command", date_cmd_buf, 'quiet=${1}']
585            if not quiet:
586                grp.rpissuing_keyword(cmd_buf)
587            status, ret_values = \
588                BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
589            if status == "PASS":
590                stdout, stderr, rc = ret_values
591                if rc == 0 and stderr == "":
592                    epoch_seconds = stdout.rstrip("\n")
593        else:
594            shell_rc, out_buf = gc.cmd_fnc_u(date_cmd_buf,
595                                             quiet=quiet,
596                                             print_output=0)
597            if shell_rc == 0:
598                epoch_seconds = out_buf.rstrip("\n")
599
600    master_req_rest = ['rest', 'host', 'requested_host', 'operating_system',
601                       'attempts_left', 'boot_progress', 'chassis',
602                       'requested_chassis' 'bmc' 'requested_bmc']
603
604    req_rest = [sub_state for sub_state in req_states if sub_state in
605                master_req_rest]
606    need_rest = (len(req_rest) > 0)
607    state = DotDict()
608    if need_rest:
609        cmd_buf = ["Read Properties", SYSTEM_STATE_URI + "enumerate",
610                   "quiet=${" + str(quiet) + "}"]
611        grp.rdpissuing_keyword(cmd_buf)
612        status, ret_values = \
613            BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
614        if status == "PASS":
615            state['rest'] = '1'
616        else:
617            state['rest'] = '0'
618
619        if int(state['rest']):
620            for url_path in ret_values:
621                for attr_name in ret_values[url_path]:
622                    # Create a state key value based on the attr_name.
623                    if type(ret_values[url_path][attr_name]) is unicode:
624                        ret_values[url_path][attr_name] = \
625                            re.sub(r'.*\.', "",
626                                   ret_values[url_path][attr_name])
627                    # Do some key name manipulations.
628                    new_attr_name = re.sub(r'^Current|(State|Transition)$',
629                                           "", attr_name)
630                    new_attr_name = re.sub(r'BMC', r'Bmc', new_attr_name)
631                    new_attr_name = re.sub(r'([A-Z][a-z])', r'_\1',
632                                           new_attr_name)
633                    new_attr_name = new_attr_name.lower().lstrip("_")
634                    new_attr_name = re.sub(r'power', r'chassis', new_attr_name)
635                    if new_attr_name in req_states:
636                        state[new_attr_name] = ret_values[url_path][attr_name]
637
638    for sub_state in req_states:
639        if sub_state in state:
640            continue
641        if sub_state.startswith("os_"):
642            # We pass "os_" requests on to get_os_state.
643            continue
644        cmd_buf = "state['" + sub_state + "'] = str(" + sub_state + ")"
645        exec(cmd_buf)
646
647    if os_host == "":
648        # The caller has not specified an os_host so as far as we're concerned,
649        # it doesn't exist.
650        return state
651
652    os_req_states = [sub_state for sub_state in req_states
653                     if sub_state.startswith('os_')]
654
655    if len(os_req_states) > 0:
656        # The caller has specified an os_host and they have requested
657        # information on os substates.
658
659        # Based on the information gathered on bmc, we'll try to make a
660        # determination of whether the os is even up.  We'll pass the result
661        # of that assessment to get_os_state to enhance performance.
662        os_up_match = DotDict()
663        for sub_state in master_os_up_match:
664            if sub_state in req_states:
665                os_up_match[sub_state] = master_os_up_match[sub_state]
666        os_up = compare_states(state, os_up_match)
667        os_state = get_os_state(os_host=os_host,
668                                os_username=os_username,
669                                os_password=os_password,
670                                req_states=os_req_states,
671                                os_up=os_up,
672                                quiet=quiet)
673        # Append os_state dictionary to ours.
674        state.update(os_state)
675
676    return state
677
678###############################################################################
679
680
681###############################################################################
682def check_state(match_state,
683                invert=0,
684                print_string="",
685                openbmc_host="",
686                openbmc_username="",
687                openbmc_password="",
688                os_host="",
689                os_username="",
690                os_password="",
691                quiet=None):
692
693    r"""
694    Check that the Open BMC machine's composite state matches the specified
695    state.  On success, this keyword returns the machine's composite state as a
696    dictionary.
697
698    Description of arguments:
699    match_state       A dictionary whose key/value pairs are "state field"/
700                      "state value".  The state value is interpreted as a
701                      regular expression.  Example call from robot:
702                      ${match_state}=  Create Dictionary  chassis=^On$
703                      ...  bmc=^Ready$
704                      ...  boot_progress=^OSStart$
705                      ${state}=  Check State  &{match_state}
706    invert            If this flag is set, this function will succeed if the
707                      states do NOT match.
708    print_string      This function will print this string to the console prior
709                      to getting the state.
710    openbmc_host      The DNS name or IP address of the BMC.
711                      This defaults to global ${OPENBMC_HOST}.
712    openbmc_username  The username to be used to login to the BMC.
713                      This defaults to global ${OPENBMC_USERNAME}.
714    openbmc_password  The password to be used to login to the BMC.
715                      This defaults to global ${OPENBMC_PASSWORD}.
716    os_host           The DNS name or IP address of the operating system.
717                      This defaults to global ${OS_HOST}.
718    os_username       The username to be used to login to the OS.
719                      This defaults to global ${OS_USERNAME}.
720    os_password       The password to be used to login to the OS.
721                      This defaults to global ${OS_PASSWORD}.
722    quiet             Indicates whether status details should be written to the
723                      console.  Defaults to either global value of ${QUIET} or
724                      to 1.
725    """
726
727    quiet = int(gp.get_var_value(quiet, 0))
728
729    grp.rprint(print_string)
730
731    req_states = match_state.keys()
732    # Initialize state.
733    state = get_state(openbmc_host=openbmc_host,
734                      openbmc_username=openbmc_username,
735                      openbmc_password=openbmc_password,
736                      os_host=os_host,
737                      os_username=os_username,
738                      os_password=os_password,
739                      req_states=req_states,
740                      quiet=quiet)
741    if not quiet:
742        gp.print_var(state)
743
744    match = compare_states(state, match_state)
745
746    if invert and match:
747        fail_msg = "The current state of the machine matches the match" +\
748                   " state:\n" + gp.sprint_varx("state", state)
749        BuiltIn().fail("\n" + gp.sprint_error(fail_msg))
750    elif not invert and not match:
751        fail_msg = "The current state of the machine does NOT match the" +\
752                   " match state:\n" +\
753                   gp.sprint_varx("state", state)
754        BuiltIn().fail("\n" + gp.sprint_error(fail_msg))
755
756    return state
757
758###############################################################################
759
760
761###############################################################################
762def wait_state(match_state=(),
763               wait_time="1 min",
764               interval="1 second",
765               invert=0,
766               openbmc_host="",
767               openbmc_username="",
768               openbmc_password="",
769               os_host="",
770               os_username="",
771               os_password="",
772               quiet=None):
773
774    r"""
775    Wait for the Open BMC machine's composite state to match the specified
776    state.  On success, this keyword returns the machine's composite state as
777    a dictionary.
778
779    Description of arguments:
780    match_state       A dictionary whose key/value pairs are "state field"/
781                      "state value".  See check_state (above) for details.
782                      This value may also be any string accepted by
783                      return_state_constant (e.g. "standby_match_state").
784                      In such a case this function will call
785                      return_state_constant to convert it to a proper
786                      dictionary as described above.
787    wait_time         The total amount of time to wait for the desired state.
788                      This value may be expressed in Robot Framework's time
789                      format (e.g. 1 minute, 2 min 3 s, 4.5).
790    interval          The amount of time between state checks.
791                      This value may be expressed in Robot Framework's time
792                      format (e.g. 1 minute, 2 min 3 s, 4.5).
793    invert            If this flag is set, this function will for the state of
794                      the machine to cease to match the match state.
795    openbmc_host      The DNS name or IP address of the BMC.
796                      This defaults to global ${OPENBMC_HOST}.
797    openbmc_username  The username to be used to login to the BMC.
798                      This defaults to global ${OPENBMC_USERNAME}.
799    openbmc_password  The password to be used to login to the BMC.
800                      This defaults to global ${OPENBMC_PASSWORD}.
801    os_host           The DNS name or IP address of the operating system.
802                      This defaults to global ${OS_HOST}.
803    os_username       The username to be used to login to the OS.
804                      This defaults to global ${OS_USERNAME}.
805    os_password       The password to be used to login to the OS.
806                      This defaults to global ${OS_PASSWORD}.
807    quiet             Indicates whether status details should be written to the
808                      console.  Defaults to either global value of ${QUIET} or
809                      to 1.
810    """
811
812    quiet = int(gp.get_var_value(quiet, 0))
813
814    if type(match_state) in (str, unicode):
815        match_state = return_state_constant(match_state)
816
817    if not quiet:
818        if invert:
819            alt_text = "cease to "
820        else:
821            alt_text = ""
822        gp.print_timen("Checking every " + str(interval) + " for up to " +
823                       str(wait_time) + " for the state of the machine to " +
824                       alt_text + "match the state shown below.")
825        gp.print_var(match_state)
826
827    if quiet:
828        print_string = ""
829    else:
830        print_string = "#"
831
832    debug = int(BuiltIn().get_variable_value("${debug}", "0"))
833    if debug:
834        # In debug we print state so no need to print the "#".
835        print_string = ""
836    check_state_quiet = 1 - debug
837    cmd_buf = ["Check State", match_state, "invert=${" + str(invert) + "}",
838               "print_string=" + print_string, "openbmc_host=" + openbmc_host,
839               "openbmc_username=" + openbmc_username,
840               "openbmc_password=" + openbmc_password, "os_host=" + os_host,
841               "os_username=" + os_username, "os_password=" + os_password,
842               "quiet=${" + str(check_state_quiet) + "}"]
843    grp.rdpissuing_keyword(cmd_buf)
844    try:
845        state = BuiltIn().wait_until_keyword_succeeds(wait_time, interval,
846                                                      *cmd_buf)
847    except AssertionError as my_assertion_error:
848        gp.printn()
849        message = my_assertion_error.args[0]
850        BuiltIn().fail(message)
851
852    if not quiet:
853        gp.printn()
854        if invert:
855            gp.print_timen("The states no longer match:")
856        else:
857            gp.print_timen("The states match:")
858        gp.print_var(state)
859
860    return state
861
862###############################################################################
863
864
865###############################################################################
866def wait_for_comm_cycle(start_boot_seconds,
867                        quiet=None):
868
869    r"""
870    Wait for communications to the BMC to stop working and then resume working.
871    This function is useful when you have initiated some kind of reboot.
872
873    Description of arguments:
874    start_boot_seconds  The time that the boot test started.  The format is the
875                        epoch time in seconds, i.e. the number of seconds since
876                        1970-01-01 00:00:00 UTC.  This value should be obtained
877                        from the BMC so that it is not dependent on any kind of
878                        synchronization between this machine and the target BMC
879                        This will allow this program to work correctly even in
880                        a simulated environment.  This value should be obtained
881                        by the caller prior to initiating a reboot.  It can be
882                        obtained as follows:
883                        state = st.get_state(req_states=['epoch_seconds'])
884    """
885
886    quiet = int(gp.get_var_value(quiet, 0))
887
888    # Validate parms.
889    error_message = gv.svalid_integer(start_boot_seconds,
890                                      var_name="start_boot_seconds")
891    if error_message != "":
892        BuiltIn().fail(gp.sprint_error(error_message))
893
894    match_state = anchor_state(DotDict([('packet_loss', '100')]))
895    # Wait for 100% packet loss trying to ping machine.
896    wait_state(match_state, wait_time="8 mins", interval="0 seconds")
897
898    match_state['packet_loss'] = '^0$'
899    # Wait for 0% packet loss trying to ping machine.
900    wait_state(match_state, wait_time="8 mins", interval="0 seconds")
901
902    # Get the uptime and epoch seconds for comparisons.  We want to be sure
903    # that the uptime is less than the elapsed boot time.  Further proof that
904    # a reboot has indeed occurred (vs random network instability giving a
905    # false positive.
906    state = get_state(req_states=['uptime', 'epoch_seconds'], quiet=quiet)
907
908    elapsed_boot_time = int(state['epoch_seconds']) - start_boot_seconds
909    gp.qprint_var(elapsed_boot_time)
910    if int(float(state['uptime'])) < elapsed_boot_time:
911        uptime = state['uptime']
912        gp.qprint_var(uptime)
913        gp.qprint_timen("The uptime is less than the elapsed boot time," +
914                        " as expected.")
915    else:
916        error_message = "The uptime is greater than the elapsed boot time," +\
917                        " which is unexpected:\n" +\
918                        gp.sprint_var(start_boot_seconds) +\
919                        gp.sprint_var(state)
920        BuiltIn().fail(gp.sprint_error(error_message))
921
922    gp.qprint_timen("Verifying that REST API interface is working.")
923    match_state = DotDict([('rest', '^1$')])
924    state = wait_state(match_state, wait_time="5 mins", interval="2 seconds")
925
926###############################################################################
927