1#!/usr/bin/env python
2
3r"""
4This module contains functions having to do with machine state: get_state,
5check_state, wait_state, etc.
6
7The 'State' is a composite of many pieces of data.  Therefore, the functions
8in this module define state as an ordered dictionary.  Here is an example of
9some test output showing machine state:
10
11default_state:
12  default_state[chassis]:                         On
13  default_state[boot_progress]:                   FW Progress, Starting OS
14  default_state[host]:                            Running
15  default_state[os_ping]:                         1
16  default_state[os_login]:                        1
17  default_state[os_run_cmd]:                      1
18
19Different users may very well have different needs when inquiring about
20state.  Support for new pieces of state information may be added to this
21module as needed.
22
23By using the wait_state function, a caller can start a boot and then wait for
24a precisely defined state to indicate that the boot has succeeded.  If
25the boot fails, they can see exactly why by looking at the current state as
26compared with the expected state.
27"""
28
29import gen_print as gp
30import gen_robot_print as grp
31import gen_valid as gv
32import gen_robot_utils as gru
33import gen_cmd as gc
34
35import commands
36from robot.libraries.BuiltIn import BuiltIn
37from robot.utils import DotDict
38
39import re
40import os
41
42# We need utils.robot to get keywords like "Get Power State".
43gru.my_import_resource("utils.robot")
44gru.my_import_resource("state_manager.robot")
45
46# The BMC code has recently been changed as far as what states are defined and
47# what the state values can be.  This module now has a means of processing both
48# the old style state (i.e. OBMC_STATES_VERSION = 0) and the new style (i.e.
49# OBMC_STATES_VERSION = 1).
50# The caller can set environment variable OBMC_STATES_VERSION to dictate
51# whether we're processing old or new style states.  If OBMC_STATES_VERSION is
52# not set it will default to 1.
53
54OBMC_STATES_VERSION = int(os.environ.get('OBMC_STATES_VERSION', 1))
55
56if OBMC_STATES_VERSION == 0:
57    # default_state is an initial value which may be of use to callers.
58    default_state = DotDict([('power', '1'),
59                             ('bmc', 'HOST_BOOTED'),
60                             ('boot_progress', 'FW Progress, Starting OS'),
61                             ('os_ping', '1'),
62                             ('os_login', '1'),
63                             ('os_run_cmd', '1')])
64    # valid_req_states, default_req_states and master_os_up_match are used by
65    # the get_state function.
66    # valid_req_states is a list of state information supported by the
67    # get_state function.
68    valid_req_states = ['ping',
69                        'packet_loss',
70                        'uptime',
71                        'epoch_seconds',
72                        'power',
73                        'bmc',
74                        'boot_progress',
75                        'os_ping',
76                        'os_login',
77                        'os_run_cmd']
78    # When a user calls get_state w/o specifying req_states, default_req_states
79    # is used as its value.
80    default_req_states = ['power',
81                          'bmc',
82                          'boot_progress',
83                          'os_ping',
84                          'os_login',
85                          'os_run_cmd']
86    # A master dictionary to determine whether the os may be up.
87    master_os_up_match = DotDict([('power', 'On'),
88                                  ('bmc', '^HOST_BOOTED$'),
89                                  ('boot_progress',
90                                   'FW Progress, Starting OS')])
91
92else:
93    # default_state is an initial value which may be of use to callers.
94    default_state = DotDict([('chassis', 'On'),
95                             ('boot_progress', 'FW Progress, Starting OS'),
96                             ('host', 'Running'),
97                             ('os_ping', '1'),
98                             ('os_login', '1'),
99                             ('os_run_cmd', '1')])
100    # valid_req_states is a list of state information supported by the
101    # get_state function.
102    # valid_req_states, default_req_states and master_os_up_match are used by
103    # the get_state function.
104    valid_req_states = ['ping',
105                        'packet_loss',
106                        'uptime',
107                        'epoch_seconds',
108                        'chassis',
109                        'boot_progress',
110                        'host',
111                        'os_ping',
112                        'os_login',
113                        'os_run_cmd']
114    # When a user calls get_state w/o specifying req_states, default_req_states
115    # is used as its value.
116    default_req_states = ['chassis',
117                          'boot_progress',
118                          'host',
119                          'os_ping',
120                          'os_login',
121                          'os_run_cmd']
122
123    # A master dictionary to determine whether the os may be up.
124    master_os_up_match = DotDict([('chassis', '^On$'),
125                                  ('boot_progress',
126                                   'FW Progress, Starting OS'),
127                                  ('host', '^Running$')])
128
129# valid_os_req_states and default_os_req_states are used by the os_get_state
130# function.
131# valid_os_req_states is a list of state information supported by the
132# get_os_state function.
133valid_os_req_states = ['os_ping',
134                       'os_login',
135                       'os_run_cmd']
136# When a user calls get_os_state w/o specifying req_states,
137# default_os_req_states is used as its value.
138default_os_req_states = ['os_ping',
139                         'os_login',
140                         'os_run_cmd']
141
142# Presently, some BMCs appear to not keep time very well.  This environment
143# variable directs the get_state function to use either the BMC's epoch time
144# or the local epoch time.
145USE_BMC_EPOCH_TIME = int(os.environ.get('USE_BMC_EPOCH_TIME', 0))
146
147
148###############################################################################
149def return_default_state():
150
151    r"""
152    Return default state dictionary.
153
154    default_state is an initial value which may be of use to callers.
155    """
156
157    return default_state
158
159###############################################################################
160
161
162###############################################################################
163def anchor_state(state):
164
165    r"""
166    Add regular expression anchors ("^" and "$") to the beginning and end of
167    each item in the state dictionary passed in.  Return the resulting
168    dictionary.
169
170    Description of Arguments:
171    state    A dictionary such as the one returned by the get_state()
172             function.
173    """
174
175    anchored_state = state.copy()
176    for key, match_state_value in anchored_state.items():
177        anchored_state[key] = "^" + str(anchored_state[key]) + "$"
178
179    return anchored_state
180
181###############################################################################
182
183
184###############################################################################
185def strip_anchor_state(state):
186
187    r"""
188    Strip regular expression anchors ("^" and "$") from the beginning and end
189    of each item in the state dictionary passed in.  Return the resulting
190    dictionary.
191
192    Description of Arguments:
193    state    A dictionary such as the one returned by the get_state()
194             function.
195    """
196
197    stripped_state = state.copy()
198    for key, match_state_value in stripped_state.items():
199        stripped_state[key] = stripped_state[key].strip("^$")
200
201    return stripped_state
202
203###############################################################################
204
205
206###############################################################################
207def compare_states(state,
208                   match_state):
209
210    r"""
211    Compare 2 state dictionaries.  Return True if they match and False if they
212    don't.  Note that the match_state dictionary does not need to have an entry
213    corresponding to each entry in the state dictionary.  But for each entry
214    that it does have, the corresponding state entry will be checked for a
215    match.
216
217    Description of arguments:
218    state           A state dictionary such as the one returned by the
219                    get_state function.
220    match_state     A dictionary whose key/value pairs are "state field"/
221                    "state value".  The state value is interpreted as a
222                    regular expression.  Every value in this dictionary is
223                    considered.  If each and every one matches, the 2
224                    dictionaries are considered to be matching.
225    """
226
227    match = True
228    for key, match_state_value in match_state.items():
229        try:
230            if not re.match(match_state_value, str(state[key])):
231                match = False
232                break
233        except KeyError:
234            match = False
235            break
236
237    return match
238
239###############################################################################
240
241
242###############################################################################
243def get_os_state(os_host="",
244                 os_username="",
245                 os_password="",
246                 req_states=default_os_req_states,
247                 os_up=True,
248                 quiet=None):
249
250    r"""
251    Get component states for the operating system such as ping, login,
252    etc, put them into a dictionary and return them to the caller.
253
254    Note that all substate values are strings.
255
256    Description of arguments:
257    os_host      The DNS name or IP address of the operating system.
258                 This defaults to global ${OS_HOST}.
259    os_username  The username to be used to login to the OS.
260                 This defaults to global ${OS_USERNAME}.
261    os_password  The password to be used to login to the OS.
262                 This defaults to global ${OS_PASSWORD}.
263    req_states   This is a list of states whose values are being requested by
264                 the caller.
265    os_up        If the caller knows that the os can't possibly be up, it can
266                 improve performance by passing os_up=False.  This function
267                 will then simply return default values for all requested os
268                 sub states.
269    quiet        Indicates whether status details (e.g. curl commands) should
270                 be written to the console.
271                 Defaults to either global value of ${QUIET} or to 1.
272    """
273
274    quiet = grp.set_quiet_default(quiet, 1)
275
276    # Set parm defaults where necessary and validate all parms.
277    if os_host == "":
278        os_host = BuiltIn().get_variable_value("${OS_HOST}")
279    error_message = gv.svalid_value(os_host, var_name="os_host",
280                                    invalid_values=[None, ""])
281    if error_message != "":
282        BuiltIn().fail(gp.sprint_error(error_message))
283
284    if os_username == "":
285        os_username = BuiltIn().get_variable_value("${OS_USERNAME}")
286    error_message = gv.svalid_value(os_username, var_name="os_username",
287                                    invalid_values=[None, ""])
288    if error_message != "":
289        BuiltIn().fail(gp.sprint_error(error_message))
290
291    if os_password == "":
292        os_password = BuiltIn().get_variable_value("${OS_PASSWORD}")
293    error_message = gv.svalid_value(os_password, var_name="os_password",
294                                    invalid_values=[None, ""])
295    if error_message != "":
296        BuiltIn().fail(gp.sprint_error(error_message))
297
298    invalid_req_states = [sub_state for sub_state in req_states
299                          if sub_state not in valid_os_req_states]
300    if len(invalid_req_states) > 0:
301        error_message = "The following req_states are not supported:\n" +\
302            gp.sprint_var(invalid_req_states)
303        BuiltIn().fail(gp.sprint_error(error_message))
304
305    # Initialize all substate values supported by this function.
306    os_ping = 0
307    os_login = 0
308    os_run_cmd = 0
309
310    if os_up:
311        if 'os_ping' in req_states:
312            # See if the OS pings.
313            cmd_buf = "ping -c 1 -w 2 " + os_host
314            if not quiet:
315                grp.rpissuing(cmd_buf)
316            rc, out_buf = commands.getstatusoutput(cmd_buf)
317            if rc == 0:
318                os_ping = 1
319
320        # Programming note: All attributes which do not require an ssh login
321        # should have been processed by this point.
322        master_req_login = ['os_login', 'os_run_cmd']
323        req_login = [sub_state for sub_state in req_states if sub_state in
324                     master_req_login]
325
326        must_login = (len([sub_state for sub_state in req_states
327                           if sub_state in master_req_login]) > 0)
328
329        if must_login:
330            # Open SSH connection to OS.
331            cmd_buf = ["SSHLibrary.Open Connection", os_host]
332            if not quiet:
333                grp.rpissuing_keyword(cmd_buf)
334            ix = BuiltIn().run_keyword(*cmd_buf)
335
336            # Login to OS.
337            cmd_buf = ["Login", os_username, os_password]
338            if not quiet:
339                grp.rpissuing_keyword(cmd_buf)
340            status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
341            if status == "PASS":
342                os_login = 1
343
344            if os_login:
345                if 'os_run_cmd' in req_states:
346                    if os_login:
347                        # Try running a simple command (uptime) on the OS.
348                        cmd_buf = ["Execute Command", "uptime",
349                                   "return_stderr=True", "return_rc=True"]
350                        if not quiet:
351                            grp.rpissuing_keyword(cmd_buf)
352                        output, stderr_buf, rc = \
353                            BuiltIn().run_keyword(*cmd_buf)
354                        if rc == 0 and stderr_buf == "":
355                            os_run_cmd = 1
356
357    os_state = DotDict()
358    for sub_state in req_states:
359        cmd_buf = "os_state['" + sub_state + "'] = str(" + sub_state + ")"
360        exec(cmd_buf)
361
362    return os_state
363
364###############################################################################
365
366
367###############################################################################
368def get_state(openbmc_host="",
369              openbmc_username="",
370              openbmc_password="",
371              os_host="",
372              os_username="",
373              os_password="",
374              req_states=default_req_states,
375              quiet=None):
376
377    r"""
378    Get component states such as power state, bmc state, etc, put them into a
379    dictionary and return them to the caller.
380
381    Note that all substate values are strings.
382
383    Description of arguments:
384    openbmc_host      The DNS name or IP address of the BMC.
385                      This defaults to global ${OPENBMC_HOST}.
386    openbmc_username  The username to be used to login to the BMC.
387                      This defaults to global ${OPENBMC_USERNAME}.
388    openbmc_password  The password to be used to login to the BMC.
389                      This defaults to global ${OPENBMC_PASSWORD}.
390    os_host           The DNS name or IP address of the operating system.
391                      This defaults to global ${OS_HOST}.
392    os_username       The username to be used to login to the OS.
393                      This defaults to global ${OS_USERNAME}.
394    os_password       The password to be used to login to the OS.
395                      This defaults to global ${OS_PASSWORD}.
396    req_states        This is a list of states whose values are being requested
397                      by the caller.
398    quiet             Indicates whether status details (e.g. curl commands)
399                      should be written to the console.
400                      Defaults to either global value of ${QUIET} or to 1.
401    """
402
403    quiet = grp.set_quiet_default(quiet, 1)
404
405    # Set parm defaults where necessary and validate all parms.
406    if openbmc_host == "":
407        openbmc_host = BuiltIn().get_variable_value("${OPENBMC_HOST}")
408    error_message = gv.svalid_value(openbmc_host,
409                                    var_name="openbmc_host",
410                                    invalid_values=[None, ""])
411    if error_message != "":
412        BuiltIn().fail(gp.sprint_error(error_message))
413
414    if openbmc_username == "":
415        openbmc_username = BuiltIn().get_variable_value("${OPENBMC_USERNAME}")
416    error_message = gv.svalid_value(openbmc_username,
417                                    var_name="openbmc_username",
418                                    invalid_values=[None, ""])
419    if error_message != "":
420        BuiltIn().fail(gp.sprint_error(error_message))
421
422    if openbmc_password == "":
423        openbmc_password = BuiltIn().get_variable_value("${OPENBMC_PASSWORD}")
424    error_message = gv.svalid_value(openbmc_password,
425                                    var_name="openbmc_password",
426                                    invalid_values=[None, ""])
427    if error_message != "":
428        BuiltIn().fail(gp.sprint_error(error_message))
429
430    # NOTE: OS parms are optional.
431    if os_host == "":
432        os_host = BuiltIn().get_variable_value("${OS_HOST}")
433        if os_host is None:
434            os_host = ""
435
436    if os_username is "":
437        os_username = BuiltIn().get_variable_value("${OS_USERNAME}")
438        if os_username is None:
439            os_username = ""
440
441    if os_password is "":
442        os_password = BuiltIn().get_variable_value("${OS_PASSWORD}")
443        if os_password is None:
444            os_password = ""
445
446    invalid_req_states = [sub_state for sub_state in req_states
447                          if sub_state not in valid_req_states]
448    if len(invalid_req_states) > 0:
449        error_message = "The following req_states are not supported:\n" +\
450            gp.sprint_var(invalid_req_states)
451        BuiltIn().fail(gp.sprint_error(error_message))
452
453    # Initialize all substate values supported by this function.
454    ping = 0
455    packet_loss = ''
456    uptime = ''
457    epoch_seconds = ''
458    power = '0'
459    chassis = ''
460    bmc = ''
461    boot_progress = ''
462    host = ''
463
464    # Get the component states.
465    if 'ping' in req_states:
466        # See if the OS pings.
467        cmd_buf = "ping -c 1 -w 2 " + openbmc_host
468        if not quiet:
469            grp.rpissuing(cmd_buf)
470        rc, out_buf = commands.getstatusoutput(cmd_buf)
471        if rc == 0:
472            ping = 1
473
474    if 'packet_loss' in req_states:
475        # See if the OS pings.
476        cmd_buf = "ping -c 5 -w 5 " + openbmc_host +\
477            " | egrep 'packet loss' | sed -re 's/.* ([0-9]+)%.*/\\1/g'"
478        if not quiet:
479            grp.rpissuing(cmd_buf)
480        rc, out_buf = commands.getstatusoutput(cmd_buf)
481        if rc == 0:
482            packet_loss = out_buf.rstrip("\n")
483
484    master_req_login = ['uptime', 'epoch_seconds']
485    req_login = [sub_state for sub_state in req_states if sub_state in
486                 master_req_login]
487
488    must_login = (len([sub_state for sub_state in req_states
489                       if sub_state in master_req_login]) > 0)
490
491    if must_login:
492        cmd_buf = ["Open Connection And Log In"]
493        if not quiet:
494            grp.rpissuing_keyword(cmd_buf)
495        BuiltIn().run_keyword(*cmd_buf)
496
497    if 'uptime' in req_states:
498        cmd_buf = ["Execute Command", "cat /proc/uptime | cut -f 1 -d ' '",
499                   "return_stderr=True", "return_rc=True"]
500        if not quiet:
501            grp.rpissuing_keyword(cmd_buf)
502        stdout_buf, stderr_buf, rc = BuiltIn().run_keyword(*cmd_buf)
503        if rc == 0 and stderr_buf == "":
504            uptime = stdout_buf
505
506    if 'epoch_seconds' in req_states:
507        date_cmd_buf = "date -u +%s"
508        if USE_BMC_EPOCH_TIME:
509            cmd_buf = ["Execute Command", date_cmd_buf, "return_stderr=True",
510                       "return_rc=True"]
511            if not quiet:
512                grp.rpissuing_keyword(cmd_buf)
513            stdout_buf, stderr_buf, rc = BuiltIn().run_keyword(*cmd_buf)
514            if rc == 0 and stderr_buf == "":
515                epoch_seconds = stdout_buf.rstrip("\n")
516        else:
517            shell_rc, out_buf = gc.cmd_fnc_u(date_cmd_buf,
518                                             quiet=1,
519                                             print_output=0)
520            if shell_rc == 0:
521                epoch_seconds = out_buf.rstrip("\n")
522
523    if 'power' in req_states:
524        cmd_buf = ["Get Power State", "quiet=${" + str(quiet) + "}"]
525        grp.rdpissuing_keyword(cmd_buf)
526        power = BuiltIn().run_keyword(*cmd_buf)
527    if 'chassis' in req_states:
528        cmd_buf = ["Get Chassis Power State", "quiet=${" + str(quiet) + "}"]
529        grp.rdpissuing_keyword(cmd_buf)
530        chassis = BuiltIn().run_keyword(*cmd_buf)
531        # Strip everything up to the final period.
532        chassis = re.sub(r'.*\.', "", chassis)
533
534    if 'bmc' in req_states:
535        if OBMC_STATES_VERSION == 0:
536            qualifier = "utils"
537        else:
538            # This will not be supported much longer.
539            qualifier = "state_manager"
540
541        cmd_buf = [qualifier + ".Get BMC State", "quiet=${" + str(quiet) + "}"]
542        grp.rdpissuing_keyword(cmd_buf)
543        bmc = BuiltIn().run_keyword(*cmd_buf)
544
545    if 'boot_progress' in req_states:
546        cmd_buf = ["Get Boot Progress", "quiet=${" + str(quiet) + "}"]
547        grp.rdpissuing_keyword(cmd_buf)
548        boot_progress = BuiltIn().run_keyword(*cmd_buf)
549
550    if 'host' in req_states:
551        if OBMC_STATES_VERSION > 0:
552            cmd_buf = ["Get Host State", "quiet=${" + str(quiet) + "}"]
553            grp.rdpissuing_keyword(cmd_buf)
554            host = BuiltIn().run_keyword(*cmd_buf)
555            # Strip everything up to the final period.
556            host = re.sub(r'.*\.', "", host)
557
558    state = DotDict()
559    for sub_state in req_states:
560        if sub_state.startswith("os_"):
561            # We pass "os_" requests on to get_os_state.
562            continue
563        cmd_buf = "state['" + sub_state + "'] = str(" + sub_state + ")"
564        exec(cmd_buf)
565
566    if os_host == "":
567        # The caller has not specified an os_host so as far as we're concerned,
568        # it doesn't exist.
569        return state
570
571    os_req_states = [sub_state for sub_state in req_states
572                     if sub_state.startswith('os_')]
573
574    if len(os_req_states) > 0:
575        # The caller has specified an os_host and they have requested
576        # information on os substates.
577
578        # Based on the information gathered on bmc, we'll try to make a
579        # determination of whether the os is even up.  We'll pass the result
580        # of that assessment to get_os_state to enhance performance.
581        os_up_match = DotDict()
582        for sub_state in master_os_up_match:
583            if sub_state in req_states:
584                os_up_match[sub_state] = master_os_up_match[sub_state]
585        os_up = compare_states(state, os_up_match)
586
587        os_state = get_os_state(os_host=os_host,
588                                os_username=os_username,
589                                os_password=os_password,
590                                req_states=os_req_states,
591                                os_up=os_up,
592                                quiet=quiet)
593        # Append os_state dictionary to ours.
594        state.update(os_state)
595
596    return state
597
598###############################################################################
599
600
601###############################################################################
602def check_state(match_state,
603                invert=0,
604                print_string="",
605                openbmc_host="",
606                openbmc_username="",
607                openbmc_password="",
608                os_host="",
609                os_username="",
610                os_password="",
611                quiet=None):
612
613    r"""
614    Check that the Open BMC machine's composite state matches the specified
615    state.  On success, this keyword returns the machine's composite state as a
616    dictionary.
617
618    Description of arguments:
619    match_state       A dictionary whose key/value pairs are "state field"/
620                      "state value".  The state value is interpreted as a
621                      regular expression.  Example call from robot:
622                      ${match_state}=  Create Dictionary  chassis=^On$
623                      ...  bmc=^Ready$
624                      ...  boot_progress=^FW Progress, Starting OS$
625                      ${state}=  Check State  &{match_state}
626    invert            If this flag is set, this function will succeed if the
627                      states do NOT match.
628    print_string      This function will print this string to the console prior
629                      to getting the state.
630    openbmc_host      The DNS name or IP address of the BMC.
631                      This defaults to global ${OPENBMC_HOST}.
632    openbmc_username  The username to be used to login to the BMC.
633                      This defaults to global ${OPENBMC_USERNAME}.
634    openbmc_password  The password to be used to login to the BMC.
635                      This defaults to global ${OPENBMC_PASSWORD}.
636    os_host           The DNS name or IP address of the operating system.
637                      This defaults to global ${OS_HOST}.
638    os_username       The username to be used to login to the OS.
639                      This defaults to global ${OS_USERNAME}.
640    os_password       The password to be used to login to the OS.
641                      This defaults to global ${OS_PASSWORD}.
642    quiet             Indicates whether status details should be written to the
643                      console.  Defaults to either global value of ${QUIET} or
644                      to 1.
645    """
646
647    quiet = grp.set_quiet_default(quiet, 1)
648
649    grp.rprint(print_string)
650
651    req_states = match_state.keys()
652    # Initialize state.
653    state = get_state(openbmc_host=openbmc_host,
654                      openbmc_username=openbmc_username,
655                      openbmc_password=openbmc_password,
656                      os_host=os_host,
657                      os_username=os_username,
658                      os_password=os_password,
659                      req_states=req_states,
660                      quiet=quiet)
661    if not quiet:
662        grp.rprint_var(state)
663
664    match = compare_states(state, match_state)
665
666    if invert and match:
667        fail_msg = "The current state of the machine matches the match" +\
668                   " state:\n" + gp.sprint_varx("state", state)
669        BuiltIn().fail("\n" + gp.sprint_error(fail_msg))
670    elif not invert and not match:
671        fail_msg = "The current state of the machine does NOT match the" +\
672                   " match state:\n" +\
673                   gp.sprint_varx("state", state)
674        BuiltIn().fail("\n" + gp.sprint_error(fail_msg))
675
676    return state
677
678###############################################################################
679
680
681###############################################################################
682def wait_state(match_state=(),
683               wait_time="1 min",
684               interval="1 second",
685               invert=0,
686               openbmc_host="",
687               openbmc_username="",
688               openbmc_password="",
689               os_host="",
690               os_username="",
691               os_password="",
692               quiet=None):
693
694    r"""
695    Wait for the Open BMC machine's composite state to match the specified
696    state.  On success, this keyword returns the machine's composite state as
697    a dictionary.
698
699    Description of arguments:
700    match_state       A dictionary whose key/value pairs are "state field"/
701                      "state value".  See check_state (above) for details.
702    wait_time         The total amount of time to wait for the desired state.
703                      This value may be expressed in Robot Framework's time
704                      format (e.g. 1 minute, 2 min 3 s, 4.5).
705    interval          The amount of time between state checks.
706                      This value may be expressed in Robot Framework's time
707                      format (e.g. 1 minute, 2 min 3 s, 4.5).
708    invert            If this flag is set, this function will for the state of
709                      the machine to cease to match the match state.
710    openbmc_host      The DNS name or IP address of the BMC.
711                      This defaults to global ${OPENBMC_HOST}.
712    openbmc_username  The username to be used to login to the BMC.
713                      This defaults to global ${OPENBMC_USERNAME}.
714    openbmc_password  The password to be used to login to the BMC.
715                      This defaults to global ${OPENBMC_PASSWORD}.
716    os_host           The DNS name or IP address of the operating system.
717                      This defaults to global ${OS_HOST}.
718    os_username       The username to be used to login to the OS.
719                      This defaults to global ${OS_USERNAME}.
720    os_password       The password to be used to login to the OS.
721                      This defaults to global ${OS_PASSWORD}.
722    quiet             Indicates whether status details should be written to the
723                      console.  Defaults to either global value of ${QUIET} or
724                      to 1.
725    """
726
727    quiet = grp.set_quiet_default(quiet, 1)
728
729    if not quiet:
730        if invert:
731            alt_text = "cease to "
732        else:
733            alt_text = ""
734        grp.rprint_timen("Checking every " + str(interval) + " for up to " +
735                         str(wait_time) + " for the state of the machine to " +
736                         alt_text + "match the state shown below.")
737        grp.rprint_var(match_state)
738
739    if quiet:
740        print_string = ""
741    else:
742        print_string = "#"
743
744    debug = int(BuiltIn().get_variable_value("${debug}", "0"))
745    if debug:
746        # In debug we print state so no need to print the "#".
747        print_string = ""
748    check_state_quiet = 1 - debug
749    cmd_buf = ["Check State", match_state, "invert=${" + str(invert) + "}",
750               "print_string=" + print_string, "openbmc_host=" + openbmc_host,
751               "openbmc_username=" + openbmc_username,
752               "openbmc_password=" + openbmc_password, "os_host=" + os_host,
753               "os_username=" + os_username, "os_password=" + os_password,
754               "quiet=${" + str(check_state_quiet) + "}"]
755    grp.rdpissuing_keyword(cmd_buf)
756    state = BuiltIn().wait_until_keyword_succeeds(wait_time, interval,
757                                                  *cmd_buf)
758    if not quiet:
759        grp.rprintn()
760        if invert:
761            grp.rprint_timen("The states no longer match:")
762        else:
763            grp.rprint_timen("The states match:")
764        grp.rprint_var(state)
765
766    return state
767
768###############################################################################
769
770
771###############################################################################
772def wait_for_comm_cycle(start_boot_seconds):
773
774    r"""
775    Wait for communications to the BMC to stop working and then resume working.
776    This function is useful when you have initiated some kind of reboot.
777
778    Description of arguments:
779    start_boot_seconds  The time that the boot test started.  The format is the
780                        epoch time in seconds, i.e. the number of seconds since
781                        1970-01-01 00:00:00 UTC.  This value should be obtained
782                        from the BMC so that it is not dependent on any kind of
783                        synchronization between this machine and the target BMC
784                        This will allow this program to work correctly even in
785                        a simulated environment.  This value should be obtained
786                        by the caller prior to initiating a reboot.  It can be
787                        obtained as follows:
788                        state = st.get_state(req_states=['epoch_seconds'])
789    """
790
791    # Validate parms.
792    error_message = gv.svalid_integer(start_boot_seconds,
793                                      var_name="start_boot_seconds")
794    if error_message != "":
795        BuiltIn().fail(gp.sprint_error(error_message))
796
797    match_state = anchor_state(DotDict([('packet_loss', '100')]))
798    # Wait for 100% packet loss trying to ping machine.
799    wait_state(match_state, wait_time="3 mins", interval="0 seconds")
800
801    match_state['packet_loss'] = '^0$'
802    # Wait for 0% packet loss trying to ping machine.
803    wait_state(match_state, wait_time="4 mins", interval="0 seconds")
804
805    # Get the uptime and epoch seconds for comparisons.  We want to be sure
806    # that the uptime is less than the elapsed boot time.  Further proof that
807    # a reboot has indeed occurred (vs random network instability giving a
808    # false positive.
809    state = get_state(req_states=['uptime', 'epoch_seconds'])
810
811    elapsed_boot_time = int(state['epoch_seconds']) - start_boot_seconds
812    grp.rprint_var(elapsed_boot_time)
813    if int(float(state['uptime'])) < elapsed_boot_time:
814        uptime = state['uptime']
815        grp.rprint_var(uptime)
816        grp.rprint_timen("The uptime is less than the elapsed boot time," +
817                         " as expected.")
818    else:
819        error_message = "The uptime is greater than the elapsed boot time," +\
820                        " which is unexpected:\n" +\
821                        gp.sprint_var(start_boot_seconds) +\
822                        gp.sprint_var(state)
823        BuiltIn().fail(gp.sprint_error(error_message))
824
825    grp.rprint_timen("Verifying that REST API interface is working.")
826    match_state = DotDict([('chassis', '.*')])
827    state = wait_state(match_state, wait_time="5 mins", interval="2 seconds")
828
829###############################################################################
830